import requests
from lxml import etree
from pymongo import MongoClient

# 链接MongoDB数据库
client = MongoClient("127.0.0.1", 27017)
collection = client["spider_month_exame"]["第三题"]

headers = {
    'User-Agent': "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
    "Cookie": "CURRENT_CITY_CODE=510100; _csrf=y8Twnz2W-zISMJwZi5w-TSCV-Cm2XS6h; Hm_lvt_4f083817a81bcb8eed537963fc1bbf10=1607586591; Hm_lpvt_4f083817a81bcb8eed537963fc1bbf10=1607586591; gr_user_id=e519f463-522f-4549-8f17-7f658db9f4c5; gr_session_id_8da2730aaedd7628=acea109f-7742-4a5c-8332-c8df9c251d7e; sajssdk_2015_cross_new_user=1; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%221764ba0b0ba9c-0f0702ccf866d9-930346c-1327104-1764ba0b0bba4%22%2C%22%24device_id%22%3A%221764ba0b0ba9c-0f0702ccf866d9-930346c-1327104-1764ba0b0bba4%22%2C%22props%22%3A%7B%22%24latest_referrer%22%3A%22%22%2C%22%24latest_referrer_host%22%3A%22%22%2C%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%7D%7D; gr_session_id_8da2730aaedd7628_acea109f-7742-4a5c-8332-c8df9c251d7e=true"
}

page = 0
for i in range(1, 6):
    page += i
    url = "http://cd.ziroom.com/z/p{}/".format(page)
    r = requests.get(url=url, headers=headers)
    html = etree.HTML(r.text)

    # 标题
    titles = html.xpath('//div[@class="info-box"]/h5/a/text()')
    # 楼层
    floods = html.xpath('//div[@class="info-box"]/div[@class="desc"]/div[1]/text()')
    # 标注
    marks = html.xpath('//div[@class="info-box"]/div[@class="desc"]/div[2]/text()')
    # 月租
    prices = html.xpath('//div[@class="item"]/div[@class="info-box"]/div[2]/span[1]/text()')

    massage_list = []
    for i in range(0, 29):
        house_list = {}
        house_list["titles"] = (titles[i])
        house_list["floods"] = (floods[i])
        house_list["marks"] = (marks[i])
        house_list["prices"] = (prices[i])
        massage_list.append(house_list)

    # print(len(massage_list))
        # 插入数据
    ret = collection.insert_many(massage_list)
